bitkeeper revision 1.1236.1.174 (424bf73dRPXTiuGP-qQmSHwGKGOyRQ)

Improved TLB flushing of subsets of CPUs. Can now do remote invlpg
as well as complete flush.
Signed-off-by: Keir Fraser <keir@xensource.com>

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 946ea05e03736379e96a548ed23bec6cdf87b899..a5648e68d5a22a7ea187ecb840513f39a4b5573e 100644 (file)
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -1476,23 +1476,34 @@ int do_mmuext_op(
             break;
     
         case MMUEXT_INVLPG_LOCAL:
-            __flush_tlb_one(op.linear_addr);
+            local_flush_tlb_one(op.linear_addr);
             break;
 
         case MMUEXT_TLB_FLUSH_MULTI:
-            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
-            break;
-    
         case MMUEXT_INVLPG_MULTI:
-            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
+        {
+            unsigned long inset = op.cpuset, outset = 0;
+            while ( inset != 0 )
+            {
+                unsigned int vcpu = find_first_set_bit(inset);
+                inset &= ~(1UL<<vcpu);
+                if ( (vcpu < MAX_VIRT_CPUS) &&
+                     ((ed = d->exec_domain[vcpu]) != NULL) )
+                    outset |= 1UL << ed->processor;
+            }
+            if ( op.cmd == MMUEXT_TLB_FLUSH_MULTI )
+                flush_tlb_mask(outset & d->cpuset);
+            else
+                flush_tlb_one_mask(outset & d->cpuset, op.linear_addr);
             break;
+        }
 
         case MMUEXT_TLB_FLUSH_ALL:
             flush_tlb_mask(d->cpuset);
             break;
     
         case MMUEXT_INVLPG_ALL:
-            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
+            flush_tlb_one_mask(d->cpuset, op.linear_addr);
             break;
 
         case MMUEXT_FLUSH_CACHE:
@@ -2029,10 +2040,10 @@ int do_update_va_mapping(unsigned long va,
         percpu_info[cpu].deferred_ops &= ~DOP_FLUSH_TLB;
         break;
     case UVMF_INVLPG_LOCAL:
-        __flush_tlb_one(va);
+        local_flush_tlb_one(va);
         break;
     case UVMF_INVLPG_ALL:
-        flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
+        flush_tlb_one_mask(d->cpuset, va);
         break;
     }
 
@@ -2317,7 +2328,7 @@ void ptwr_flush(const int which)
 
     /* Ensure that there are no stale writable mappings in any TLB. */
     /* NB. INVLPG is a serialising instruction: flushes pending updates. */
-    __flush_tlb_one(l1va); /* XXX Multi-CPU guests? */
+    local_flush_tlb_one(l1va); /* XXX Multi-CPU guests? */
     PTWR_PRINTK("[%c] disconnected_l1va at %p now %p\n",
                 PTWR_PRINT_WHICH, ptep, pte);
 
@@ -2636,7 +2647,7 @@ int ptwr_do_page_fault(unsigned long addr)
          likely(!shadow_mode_enabled(ed->domain)) )
     {
         *pl2e = mk_l2_pgentry(l2e & ~_PAGE_PRESENT);
-        flush_tlb(); /* XXX Multi-CPU guests? */
+        local_flush_tlb(); /* XXX Multi-CPU guests? */
     }
     
     /* Temporarily map the L1 page, and make a copy of it. */

diff --git a/xen/arch/x86/mtrr/generic.c b/xen/arch/x86/mtrr/generic.c
index 26000d4b495c05f4a46d45b58c95a3d40e81462e..e8d990aec79d1238dfd945e527acce9396e31fc2 100644 (file)
--- a/xen/arch/x86/mtrr/generic.c
+++ b/xen/arch/x86/mtrr/generic.c
@@ -261,7 +261,7 @@ static void prepare_set(void)
        }
 
        /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
-       __flush_tlb();
+       local_flush_tlb();
 
        /*  Save MTRR state */
        rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
@@ -273,7 +273,7 @@ static void prepare_set(void)
 static void post_set(void)
 {
        /*  Flush TLBs (no need to flush caches - they are disabled)  */
-       __flush_tlb();
+       local_flush_tlb();
 
        /* Intel (P6) standard MTRRs */
        wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);

diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c
index 962417238fc6eee0673edca89cf653290c3731aa..023b8fcab58601c8b9735b42e4ae457e121ee1b7 100644 (file)
--- a/xen/arch/x86/smp.c
+++ b/xen/arch/x86/smp.c
@@ -148,17 +148,20 @@ static inline void send_IPI_allbutself(int vector)
 }
 
 static spinlock_t flush_lock = SPIN_LOCK_UNLOCKED;
-static unsigned long flush_cpumask;
+static unsigned long flush_cpumask, flush_va;
 
 asmlinkage void smp_invalidate_interrupt(void)
 {
     ack_APIC_irq();
     perfc_incrc(ipis);
-    local_flush_tlb();
+    if ( flush_va == FLUSHVA_ALL )
+        local_flush_tlb();
+    else
+        local_flush_tlb_one(flush_va);
     clear_bit(smp_processor_id(), &flush_cpumask);
 }
 
-void flush_tlb_mask(unsigned long mask)
+void __flush_tlb_mask(unsigned long mask, unsigned long va)
 {
     ASSERT(local_irq_is_enabled());
     
@@ -172,6 +175,7 @@ void flush_tlb_mask(unsigned long mask)
     {
         spin_lock(&flush_lock);
         flush_cpumask = mask;
+        flush_va      = va;
         send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
         while ( flush_cpumask != 0 )
             cpu_relax();
@@ -190,6 +194,7 @@ void new_tlbflush_clock_period(void)
         spin_lock(&flush_lock);
         flush_cpumask  = (1UL << smp_num_cpus) - 1;
         flush_cpumask &= ~(1UL << smp_processor_id());
+        flush_va       = FLUSHVA_ALL;
         send_IPI_allbutself(INVALIDATE_TLB_VECTOR);
         while ( flush_cpumask != 0 )
             cpu_relax();
@@ -203,13 +208,13 @@ void new_tlbflush_clock_period(void)
 
 static void flush_tlb_all_pge_ipi(void *info)
 {
-    __flush_tlb_pge();
+    local_flush_tlb_pge();
 }
 
 void flush_tlb_all_pge(void)
 {
     smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1);
-    __flush_tlb_pge();
+    local_flush_tlb_pge();
 }
 
 void smp_send_event_check_mask(unsigned long cpu_mask)

diff --git a/xen/arch/x86/x86_32/mm.c b/xen/arch/x86/x86_32/mm.c
index ac1f10def532098fbf25cead274530874cef80d8..c33758b69dfcf30c5bb13eaeb1f533fad4150249 100644 (file)
--- a/xen/arch/x86/x86_32/mm.c
+++ b/xen/arch/x86/x86_32/mm.c
@@ -48,7 +48,7 @@ int map_pages(
         {
             /* Super-page mapping. */
             if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) )
-                __flush_tlb_pge();
+                local_flush_tlb_pge();
             *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE);
 
             v += 1 << L2_PAGETABLE_SHIFT;
@@ -66,7 +66,7 @@ int map_pages(
             }
             pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v);
             if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) )
-                __flush_tlb_one(v);
+                local_flush_tlb_one(v);
             *pl1e = mk_l1_pgentry(p|flags);
 
             v += 1 << L1_PAGETABLE_SHIFT;

diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
index d5e925492507b6f09d0e53f8aa665f1df42fca55..2d19d6f85f578a9c3cae42172464027bf95aff76 100644 (file)
--- a/xen/arch/x86/x86_64/mm.c
+++ b/xen/arch/x86/x86_64/mm.c
@@ -89,7 +89,7 @@ int map_pages(
         {
             /* Super-page mapping. */
             if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) )
-                __flush_tlb_pge();
+                local_flush_tlb_pge();
             *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE);
 
             v += 1 << L2_PAGETABLE_SHIFT;
@@ -107,7 +107,7 @@ int map_pages(
             }
             pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v);
             if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) )
-                __flush_tlb_one(v);
+                local_flush_tlb_one(v);
             *pl1e = mk_l1_pgentry(p|flags);
 
             v += 1 << L1_PAGETABLE_SHIFT;

diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c
index fceee60984aac8633c689ccce22886eed7f77498..7cd543b834905f67970791552610ef05c7c316fb 100644 (file)
--- a/xen/common/grant_table.c
+++ b/xen/common/grant_table.c
@@ -102,7 +102,8 @@ __gnttab_map_grant_ref(
     if ( ((host_virt_addr != 0) || (flags & GNTMAP_host_map) ) &&
          unlikely(!__addr_ok(host_virt_addr)))
     {
-        DPRINTK("Bad virtual address (%x) or flags (%x).\n", host_virt_addr, flags);
+        DPRINTK("Bad virtual address (%x) or flags (%x).\n",
+                host_virt_addr, flags);
         (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
         return GNTST_bad_gntref;
     }
@@ -332,8 +333,10 @@ __gnttab_map_grant_ref(
          */
     }
 
-    /* Only make the maptrack live _after_ writing the pte, in case
-     * we overwrite the same frame number, causing a maptrack walk to find it */
+    /*
+     * Only make the maptrack live _after_ writing the pte, in case we 
+     * overwrite the same frame number, causing a maptrack walk to find it.
+     */
     ld->grant_table->maptrack[handle].domid         = dom;
     ld->grant_table->maptrack[handle].ref_and_flags =
         (ref << MAPTRACK_REF_SHIFT) | (flags & MAPTRACK_GNTMAP_MASK);
@@ -364,13 +367,14 @@ gnttab_map_grant_ref(
     unsigned long va = 0;
 
     for ( i = 0; i < count; i++ )
-        if ( __gnttab_map_grant_ref(&uop[i], &va) == 0)
+        if ( __gnttab_map_grant_ref(&uop[i], &va) == 0 )
             flush++;
 
+    /* XXX KAF: I think we are probably flushing too much here. */
     if ( flush == 1 )
-        __flush_tlb_one(va);
+        flush_tlb_one_mask(current->domain->cpuset, va);
     else if ( flush != 0 )
-        local_flush_tlb();
+        flush_tlb_mask(current->domain->cpuset);
 
     return 0;
 }
@@ -457,7 +461,7 @@ __gnttab_unmap_grant_ref(
         unsigned long   _ol1e;
 
         pl1e = &linear_pg_table[l1_linear_offset(virt)];
-                                                                                            
+
         if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
         {
             DPRINTK("Could not find PTE entry for address %x\n", virt);
@@ -526,14 +530,13 @@ gnttab_unmap_grant_ref(
     unsigned long va = 0;
 
     for ( i = 0; i < count; i++ )
-        if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0)
+        if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 )
             flush++;
 
     if ( flush == 1 )
-        __flush_tlb_one(va);
-
+        flush_tlb_one_mask(current->domain->cpuset, va);
     else if ( flush != 0 )
-        local_flush_tlb();
+        flush_tlb_mask(current->domain->cpuset);
 
     return 0;
 }

diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h
index 5958b5f524eb833f1aa15667ad98c1b83b7b0a43..8f48465cb211c8c7286686ded9f7024afb83bd92 100644 (file)
--- a/xen/include/asm-x86/flushtlb.h
+++ b/xen/include/asm-x86/flushtlb.h
@@ -77,46 +77,35 @@ static inline unsigned long read_cr3(void)
 /* Write pagetable base and implicitly tick the tlbflush clock. */
 extern void write_cr3(unsigned long cr3);
 
-/*
- * TLB flushing:
- *
- *  - flush_tlb() flushes the current mm struct TLBs
- *  - flush_tlb_all() flushes all processes TLBs
- *  - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
- *
- * ..but the i386 has somewhat limited tlb flushing capabilities,
- * and page-granular flushes are available only on i486 and up.
- */
-
-#define __flush_tlb()                                             \
+#define local_flush_tlb()                                         \
     do {                                                          \
         unsigned long cr3 = read_cr3();                           \
         write_cr3(cr3);                                           \
     } while ( 0 )
 
-#ifndef CONFIG_SMP
+#define local_flush_tlb_pge()                                     \
+    do {                                                          \
+        __pge_off();                                              \
+        local_flush_tlb();                                        \
+        __pge_on();                                               \
+    } while ( 0 )
 
-#define flush_tlb()               __flush_tlb()
-#define flush_tlb_all()           __flush_tlb()
-#define flush_tlb_all_pge()       __flush_tlb_pge()
-#define local_flush_tlb()         __flush_tlb()
-#define flush_tlb_cpu(_cpu)       __flush_tlb()
-#define flush_tlb_mask(_mask)     __flush_tlb()
-#define try_flush_tlb_mask(_mask) __flush_tlb()
+#define local_flush_tlb_one(__addr) \
+    __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
 
-#else
+#define flush_tlb_all()     flush_tlb_mask((1 << smp_num_cpus) - 1)
 
+#ifndef CONFIG_SMP
+#define flush_tlb_all_pge()          local_flush_tlb_pge()
+#define flush_tlb_mask(_mask)        local_flush_tlb()
+#define flush_tlb_one_mask(_mask,_v) local_flush_tlb_one(_v)
+#else
 #include <xen/smp.h>
-
-extern int try_flush_tlb_mask(unsigned long mask);
-extern void flush_tlb_mask(unsigned long mask);
+#define FLUSHVA_ALL (~0UL)
 extern void flush_tlb_all_pge(void);
-
-#define flush_tlb()        __flush_tlb()
-#define flush_tlb_all()     flush_tlb_mask((1 << smp_num_cpus) - 1)
-#define local_flush_tlb()   __flush_tlb()
-#define flush_tlb_cpu(_cpu) flush_tlb_mask(1 << (_cpu))
-
+extern void __flush_tlb_mask(unsigned long mask, unsigned long va);
+#define flush_tlb_mask(_mask)        __flush_tlb_mask(_mask,FLUSHVA_ALL)
+#define flush_tlb_one_mask(_mask,_v) __flush_tlb_mask(_mask,_v)
 #endif
 
 #endif /* __FLUSHTLB_H__ */

diff --git a/xen/include/asm-x86/page.h b/xen/include/asm-x86/page.h
index 78e2648b5465b50dfe7a5f44d6e7e5880d9eb399..58f67d88b8ba103ed4e7f3a00a9071ba7fb3b5fb 100644 (file)
--- a/xen/include/asm-x86/page.h
+++ b/xen/include/asm-x86/page.h
@@ -78,8 +78,6 @@ extern root_pgentry_t idle_pg_table[ROOT_PAGETABLE_ENTRIES];
 
 extern void paging_init(void);
 
-/* Flush global pages as well. */
-
 #define __pge_off()                                                     \
     do {                                                                \
         __asm__ __volatile__(                                           \
@@ -94,17 +92,6 @@ extern void paging_init(void);
             : : "r" (mmu_cr4_features) );                               \
     } while ( 0 )
 
-
-#define __flush_tlb_pge()                                               \
-    do {                                                                \
-        __pge_off();                                                    \
-        __flush_tlb();                                                  \
-        __pge_on();                                                     \
-    } while ( 0 )
-
-#define __flush_tlb_one(__addr) \
-    __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
-
 #endif /* !__ASSEMBLY__ */